/*
 * Javlov - a Java toolkit for reinforcement learning with multi-agent support.
 * 
 * Copyright (c) 2009 Matthijs Snel
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package net.javlov;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class Simulator implements Runnable {

	public static final int	STATE_PLAY = 0,
							STATE_STEP = 1;
	
	/**
	 * Used for suspending the simulator.
	 */
	private boolean suspended = false;
	private int state = STATE_PLAY;
	
	protected Agent agnt;
	protected Environment env;
	
	/**
	 * For how many steps the agents should be evaluated (continuing tasks)
	 */
	protected int steps;
	
	/**
	 * Flags end of episode (episodic tasks)
	 */
	protected boolean episodeEnd;
	
	/**
	 * List of statistics to update on each step.
	 */
	protected List<Statistic> stats;
	
	/**
	 * Reward received per step.
	 */
	protected double reward;
	
	/**
	 * Last State and Action, to avoid having to create them again on each step
	 */
	private State s;
	private Action act;
	
	public Simulator() {
		steps = 0;
		reward = 0;
		stats = new ArrayList<Statistic>();
	}
	
	public Simulator( Agent a, Environment env ) {
		this();
		setAgent(a);
		setEnvironment(env);
	}
	
	public void setAgent( Agent a ) {
		agnt = a;
	}
	
	public void addStatistic(Statistic stat) {
		stats.add(stat);
	}
	
	public void step() {
		s = env.getObservation(agnt);
		episodeEnd = s.isTerminal();
		//System.out.println(s);
		if ( episodeEnd ) {
			agnt.lastStep(s, reward);
			return;
		}
		act = agnt.doStep(s, reward);
		reward = env.executeAction(act, agnt);
		//System.out.println(act + "-" + act.getID() + ", reward: " + reward);
		for ( Statistic stat : stats )
			stat.update(s, act, reward);
		//System.out.println("-------------------------");
	}
	
	public void firstStep() {
		s = env.getObservation(agnt);
		episodeEnd = s.isTerminal();
		//System.out.println(s);
		if ( episodeEnd ) {
			agnt.lastStep(s, 0); //not really possible, but anyway
			return;
		}
		act = agnt.firstStep(s);
		reward = env.executeAction(act, agnt);
		//System.out.println(act + "-" + act.getID() + ", reward: " + reward);
		for ( Statistic stat : stats )
			stat.update(s, act, reward);
		//System.out.println("-------------------------");
	}
	
	public boolean isSuspended() {
		return suspended;
	}
	
	public void init() {
		episodeEnd = false;
		env.init();
		agnt.init();
		for ( Statistic stat : stats )
			stat.init();
	}
	
	public void reset() {
		episodeEnd = false;
		env.reset();
		agnt.reset();
		for ( Statistic stat : stats )
			stat.reset();
	}
	
	@Override
	public void run() {
		firstStep();
		for ( int i = 0; i < steps-1; i++ ) {
			while ( suspended ) {
				synchronized(this) {
					try {
						wait();
					} catch ( InterruptedException e) {}
				}
			}
			step();
			
			if ( state == STATE_STEP )
				suspended = true;
		}
	}
	
	public void runEpisode() {
		firstStep();
		while ( !episodeEnd ) {
			while ( suspended ) {
				synchronized(this) {
					try {
						wait();
					} catch ( InterruptedException e) {}
				}
			}
			step();
			
			if ( state == STATE_STEP )
				suspended = true;
		}
	}
	
	public void runEpisodes(int episodes) {
		for ( int i = 0; i < episodes; i++ ) {
			runEpisode();
			reset();
			//suspend();
		}
	}
		
	
	public void setEnvironment( Environment env ) {
		this.env = env;
	}
	
	public void setEvaluationSteps(int steps) {
		this.steps = steps;
	}
	
	public void setState(int state) {
		this.state = state;
	}
	
	public void playSteps(boolean steps) {
		if ( steps )
			state = STATE_STEP;
		else
			state = STATE_PLAY;
	}
	
	public void suspend() {
		suspended = !suspended;
		if ( !suspended ) {
			synchronized(this) {
				notifyAll();
			}
		}
	}
	
	public boolean episodeEnd() {
		return episodeEnd;
	}
}
